package com.atguigu.bigdata.spark.core.rdd.dependency

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

/**
  * create by undeRdoG on  2021-06-14  13:21
  * 凡心所向，素履以往，生如逆旅，一苇以航。
  */
object Spark01_RDD_Dep {


  /**
    * (1) datas/word.txt MapPartitionsRDD[1] at textFile at Spark01_RDD_Dep.scala:16 []
    * |  datas/word.txt HadoopRDD[0] at textFile at Spark01_RDD_Dep.scala:16 []
    * *******************************************************************************************
    * (1) MapPartitionsRDD[2] at flatMap at Spark01_RDD_Dep.scala:22 []
    * |  datas/word.txt MapPartitionsRDD[1] at textFile at Spark01_RDD_Dep.scala:16 []
    * |  datas/word.txt HadoopRDD[0] at textFile at Spark01_RDD_Dep.scala:16 []
    * *******************************************************************************************
    * (1) MapPartitionsRDD[3] at map at Spark01_RDD_Dep.scala:28 []
    * |  MapPartitionsRDD[2] at flatMap at Spark01_RDD_Dep.scala:22 []
    * |  datas/word.txt MapPartitionsRDD[1] at textFile at Spark01_RDD_Dep.scala:16 []
    * |  datas/word.txt HadoopRDD[0] at textFile at Spark01_RDD_Dep.scala:16 []
    * *******************************************************************************************
    *
    **/

  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(sparkConf)

    val lines: RDD[String] = sc.textFile("datas/word.txt")

    //  打印血缘关系
    println(lines.toDebugString)
    println("*******************************************************************************************")

    val words: RDD[String] = lines.flatMap(line => line.split(" "))

    //  打印血缘关系
    println(words.toDebugString)
    println("*******************************************************************************************")

    val wordToOne: RDD[(String, Int)] = words.map(word => (word, 1))

    //  打印血缘关系
    println(wordToOne.toDebugString)
    println("*******************************************************************************************")
    val res: RDD[(String, Int)] = wordToOne.reduceByKey(_ + _)

    res.collect().foreach(println)
  }

}
